package com.chance.cc.crawler.development.scripts.stats;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.crawler.CrawlerResultTags;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;

public class StatsCrawlerScript extends CrawlerCommonScript {

    private Logger log= LoggerFactory.getLogger(StatsCrawlerScript.class);

    public static final String listUrlPrefix = "http://www.stats.gov.cn/tjsj/zxfb/";

    @Override
    public String domain() {
        return "stats";
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        return true;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular("http://www.stats.gov.cn/tjsj/\\w*/");//初始页
        addUrlRegular("http://www.stats.gov.cn/tjsj/\\w*/\\S*.html");//翻页
        addUrlRegular("http://www.stats.gov.cn/tjsj/\\w*/\\d+/\\S*.html");//详情页
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks=new ArrayList<>();
            String requestUrl=httpPage.getRequest().getUrl();
            String nextPageUrl = "";
            if (requestUrl.matches("http://www.stats.gov.cn/tjsj/zxfb/")){
                nextPageUrl = requestUrl + "index_1.html";
            }else {
                int pn = Integer.parseInt(requestUrl.substring(requestUrl.lastIndexOf("_") + 1, requestUrl.lastIndexOf(".")));
                nextPageUrl = "http://www.stats.gov.cn/tjsj/zxfb/index_" + (pn + 1) + ".html";
            }
            //解析下一页的链接
//            String nextUrl = listUrlPrefix+httpPage.getHtml().xpath("//dl[@class=\"fenye\"]/a/@href").get();
            CrawlerRequestRecord turnRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .build();
            parsedLinks.add(turnRequest);

            //详细页解析
            List<Selectable> nodes = httpPage.getHtml().xpath("//div[@class=\"center_list\"]/ul/li").nodes();

            for (Selectable node:nodes){
                String itemUrl=node.xpath("./a/@href").get();
                    if (StringUtils.isBlank(itemUrl)){
                        continue;
                    }

                    if(itemUrl.contains("tjsj")){
                        itemUrl = "http://www.stats.gov.cn" + itemUrl;
                    }else{
                        itemUrl = listUrlPrefix + itemUrl.substring(2);
                    }

                    //获取时间
                    String pubTime = node.xpath(".//font[@class=\"cont_tit02\"]/text()").get();
                    if (StringUtils.isBlank(pubTime)){
                        continue;
                    }
                try {
                    long releaseTime = DateUtils.parseDate(pubTime, "yyyy-MM-dd").getTime();

                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(itemUrl)
                            .httpUrl(itemUrl)
                            .releaseTime(releaseTime)
                            .resultLabelTag(article)
                            .build();

                    parsedLinks.add(itemRecord);
                } catch (ParseException e) {
                    e.printStackTrace();
                }

            }

        return parsedLinks;
    }



    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        CrawlerResultTags crawlerResultTags = crawlerRecord.tagsCreator().resultTags();
        if (crawlerResultTags.hasDataType(article)) {
            crawlerDataList.add(washArticle(crawlerRecord, page));

        }

        return crawlerDataList;
    }

    public CrawlerData washArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        try {

            String releaseTime = "";
            List<String> all = httpPage.getHtml().xpath("//font[@class=\"xilan_titf\"]/font//text()").all();
            for (String data : all) {
                data = StringUtils.isNotEmpty(data) ? getString("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}",data) : "";
                if(StringUtils.isNotEmpty(data)){
                    releaseTime = data;
                    break;
                }
            }


            List<String> texts = httpPage.getHtml().xpath("//div[@class=\"TRS_PreAppend\"]/p//text()").all();
            StringBuffer conents = new StringBuffer();
            for (String text : texts) {
                conents.append(text).append(" ");
            }

            String itemUrl = httpPage.getRequest().getUrl();
            String articleKey = itemUrl.substring(itemUrl.lastIndexOf("_") + 1,itemUrl.lastIndexOf("."));

            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(itemUrl)
                    .releaseTime(DateUtils.parseDate(releaseTime,"yyyy-MM-dd HH:mm").getTime())
                    .addContentKV("content",conents.toString().trim())
                    .addContentKV("title",httpPage.getHtml().xpath("//h2[@class=\"xilan_tit\"]").get())
                    .addContentKV("author",httpPage.getHtml().xpath("//font[@class=\"xilan_titf\"]/font/font").get())
                    .build();
            return crawlerData;
        } catch (ParseException e) {
            log.error(e.getMessage(),e);
        }
        return null;

    }

    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {

    }

    private static String getString(String regx, String input) {
        Pattern compile = Pattern.compile(regx);
        Matcher matcher = compile.matcher(input);
        List<String> urlList = new ArrayList<>();
        while (matcher.find()) {
            return matcher.group(0);
        }
        return null;
    }
}
